import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import re
from python_ai.common.xcommon import sep
pd.set_option('display.max_columns', None, 'display.max_colwidth', 1000, 'display.expand_frame_repr', False)
plt.rcParams['font.sans-serif'] = ['Simhei']
plt.rcParams['axes.unicode_minus'] = False

spr = 2
spc = 3
spn = 0
plt.figure(figsize=[12, 8])

sep('Data')
df = pd.read_csv(r'../../../../../large_data/ML2/train_titanic.csv')
m, n = df.shape
np.random.seed(666)
idx_rnd = np.random.permutation(m)
df = df.iloc[idx_rnd]
print(df.shape)
print(df[:5])

sep('check na')
sum_na = df.isnull().sum(axis=0)
idx_na = sum_na != 0
print(sum_na[idx_na])
print(df.columns[idx_na])

sep('Check and remove PassengerId')
# print(df['PassengerId'].value_counts())
print('Doubled value count:')
print(df['PassengerId'].value_counts().value_counts())
df.drop(labels='PassengerId', axis=1, inplace=True)

sep('Analyse Pclass')
vcnt_pclass = df['Pclass'].value_counts().sort_index()
idx_survived = df['Survived'] == 1
idx_dead = np.invert(idx_survived)
vcnt_survived_pclass = df.loc[idx_survived, 'Pclass'].value_counts().sort_index()
vcnt_dead_pclass = df.loc[idx_dead, 'Pclass'].value_counts().sort_index()
df_vcnt_pclass_survived_or_dead = pd.DataFrame({'Survived': vcnt_survived_pclass,
                                               'Dead': vcnt_dead_pclass}).sort_index()
spn += 1
ax = plt.subplot(spr, spc, spn)
plt.title('Pclass')
df_vcnt_pclass_survived_or_dead.sort_index().plot(ax=ax, kind='bar')
spn += 1
ax = plt.subplot(spr, spc, spn)
plt.title('Pclass seaborn')
df['survived_label'] = df['Survived'].map({0: 'yes', 1: 'no'})
sns.countplot(data=df,
              x='Pclass',
              hue='survived_label',
              hue_order=['yes', 'no'],
              ax=ax,
              )
print('count Pclass:')
print(vcnt_pclass)
print('count Pclass: survived:')
print(vcnt_survived_pclass)
print('count Pclass: dead:')
print(vcnt_dead_pclass)
print('count Pclass: survived rate:')
print((vcnt_survived_pclass/vcnt_pclass).sort_index())

sep('Analyse name')


def x_extract_name(name):
    if not hasattr(x_extract_name, 'regexp'):
        x_extract_name.regexp = re.compile(r'^[^,]+,\s*([^\.]+)\.')
    matcher = x_extract_name.regexp.match(name)
    if matcher is None:
        return ''
    else:
        return matcher[1]


df['Name'] = df['Name'].map(x_extract_name)
df.loc[~(df['Name'].isin(['Mr', 'Mrs', 'Miss', 'Master'])), 'Name'] = 'Rare'
print(df['Name'].value_counts())
vcnt_name = df['Name'].value_counts().sort_index()
vcnt_name_survived = df.loc[idx_survived, 'Name'].value_counts().sort_index()
vcnt_name_dead = df.loc[idx_dead, 'Name'].value_counts().sort_index()
df_vcnt_name_survived_or_dead = pd.DataFrame({'Survived': vcnt_name_survived,
                                                'Dead': vcnt_name_dead}).sort_index()
spn += 1
ax = plt.subplot(spr, spc, spn)
plt.title('Name')
df_vcnt_name_survived_or_dead.plot(kind='bar', ax=ax)

sep('Analyse sex')
vcnt_sex = df['Sex'].value_counts().sort_index()
vcnt_sex_survived = df.loc[idx_survived, 'Sex'].value_counts().sort_index()
vcnt_sex_dead = df.loc[idx_dead, 'Sex'].value_counts().sort_index()
df_vcnt_sex_survived_or_dead = pd.DataFrame({'Survived': vcnt_sex_survived,
                                               'Dead': vcnt_sex_dead})
spn += 1
ax = plt.subplot(spr, spc, spn)
plt.title('Sex')
df_vcnt_sex_survived_or_dead.plot(kind='bar', ax=ax)

sep('Fillna Fare')
# Fill
df['Fare'].fillna(df['Fare'].mean(), inplace=True)
# Check
sum_na = df.isnull().sum(axis=0)
idx_na = sum_na != 0
print(sum_na[idx_na])
print(df.columns[idx_na])

sep('Fillna Embarked')
# Fill
print(df['Embarked'].value_counts())
embarked_most_freq = df['Embarked'].value_counts().index[0]
print(embarked_most_freq)
df['Embarked'].fillna(embarked_most_freq, inplace=True)
# Check
sum_na = df.isnull().sum(axis=0)
idx_na = sum_na != 0
print(sum_na[idx_na])
print(df.columns[idx_na])

sep('Drop Cabin')
df.drop(labels=['Cabin'], axis=1, inplace=True)
# Check
sum_na = df.isnull().sum(axis=0)
idx_na = sum_na != 0
print(sum_na[idx_na])
print(df.columns[idx_na])

sep('Hypothesis of age')
df_labels = df[['Survived', 'Pclass', 'Sex', 'Embarked', 'Name']]
df_continuous = df[['Fare']]  # ATTENTION to get a matrix, not vector
from sklearn.preprocessing import OneHotEncoder, StandardScaler
enc = OneHotEncoder()
std = StandardScaler()
x_labels = enc.fit_transform(df_labels).A
x_continuous = std.fit_transform(df_continuous)
print(type(df_labels))
print(type(df_continuous))
x = np.c_[x_labels, x_continuous]
idx_age_na = df['Age'].isna()
idx_age_notna = np.invert(idx_age_na)
x_train = x[idx_age_notna, :]
x_test = x[idx_age_na, :]
y_train = df.loc[idx_age_notna, 'Age']
from sklearn.linear_model import Ridge
from sklearn.model_selection import GridSearchCV
estimator = Ridge()
params = dict(alpha=[0.1, 0.3, 0.5, 0.7, 1, 1.5, 2, 5])
grid = GridSearchCV(estimator, params, cv=5)
grid.fit(x_train, y_train)
print(f'Best score = {grid.best_score_}')
print(f'Best params = {grid.best_params_}')
model = Ridge(**(grid.best_params_))
model.fit(x_train, y_train)
df.loc[idx_age_na, 'Age'] = model.predict(x_test)
# Check
sum_na = df.isnull().sum(axis=0)
idx_na = sum_na != 0
print(sum_na[idx_na])
print(df.columns[idx_na])

sep('Age sections')
df.loc[df['Age'] <= 12, 'age_section'] = 'child'
df.loc[(df['Age'] > 12) & (df['Age'] <= 40), 'age_section'] = 'adult'
df.loc[df['Age'] > 40, 'age_section'] = 'elder'
vcnt_age_sec = df['age_section'].value_counts().sort_index()
print(vcnt_age_sec)
vcnt_age_sec_survived = df.loc[idx_survived, 'age_section'].value_counts().sort_index()
vcnt_age_sec_dead = df.loc[idx_dead, 'age_section'].value_counts().sort_index()
df_vcnt_age_sec_survived_or_dead = pd.DataFrame({'Survived': vcnt_age_sec_survived,
                                                 'Dead': vcnt_age_sec_dead})
spn += 1
ax = plt.subplot(spr, spc, spn)
plt.title('Age Section')
df_vcnt_age_sec_survived_or_dead.plot(kind='bar', ax=ax)

# Finally show all plotting.
plt.show()
